Introduction

In this notebook we train the dynamic part of the cross-domain model. The dynamic maps in the Carla dataset are the night, evening, and rainy routes. We train this by using a Siamese network and triplet loss. The non-grey parts of following image denote the dynamic network as a part of the bigger model. Trainig of the dynamic model occurs in a rotating fashion. (For the code please check networks/crossdomain_dynamic.py)

Setup

import gc
gc.collect()

209

import os
import cv2
import torch
import csv
import torchvision.models as models
import torchvision
import torch.nn as nn
import torch.utils
from torch.utils.data import Dataset, DataLoader
import torch.optim as optim
import torch.optim.lr_scheduler as lr_scheduler
from PIL import Image
from torchvision import transforms
import matplotlib.pyplot as plt
import albumentations as albu
import pandas as pd
import numpy as np
import time
import h5py
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
import random
import re
import plotly.express as px
from plotly.offline import init_notebook_mode, iplot
from tqdm.notebook import tqdm
import shutil

from utils.utility import *
from datasets.crossdomain_dataset import *
from utils.augmentations import *
from networks.siamese import *
from networks.crossdomain_dynamic import *

import warnings
warnings.filterwarnings("ignore")

init_notebook_mode()
pd.set_option('max_colwidth', 600)
%matplotlib inline

Setting paths

path_root = 'data'
path_h5 = os.path.join(path_root, 'h5')
path_noon = os.path.join(path_h5, 'noon.hdf5')
path_night = os.path.join(path_h5, 'night.hdf5')
path_evening = os.path.join(path_h5, 'sunset.hdf5')
path_rainy_noon = os.path.join(path_h5, 'rainy_noon.hdf5')

path_noon_2 = os.path.join(path_h5, 'noon2.hdf5')
path_night_2 = os.path.join(path_h5, 'night2.hdf5')
path_evening_2 = os.path.join(path_h5, 'sunset2.hdf5')
path_rainy_noon_2 = os.path.join(path_h5, 'rainy_noon2.hdf5')

path_save = os.path.join(path_root, 'saved')
path_models = os.path.join(path_save, 'models')
path_dump = os.path.join(path_save, 'logs')
path_embeddings = os.path.join(path_save, 'embeddings')

# For reproducibility
SEED = 42
torch.manual_seed(SEED)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(SEED)
random.seed(SEED)

Loading image arrays and precalculated statistics

arr_noon_images = get_image_arr(path_noon)
arr_night_images = get_image_arr(path_night)
arr_evening_images = get_image_arr(path_evening)
arr_rainy_noon_images = get_image_arr(path_rainy_noon)

arr_noon_2_images = get_image_arr(path_noon_2)
arr_night_2_images = get_image_arr(path_night_2)
arr_evening_2_images = get_image_arr(path_evening_2)
arr_rainy_noon_2_images = get_image_arr(path_rainy_noon_2)

len_ds = arr_noon_images.shape[0]

# mean, std
stats_noon = np.array([[0.47357196, 0.47686451, 0.48483694], [0.26173203, 0.26308321, 0.2692394]])   
stats_night = np.array([[0.13137933, 0.11963478, 0.10502443], [0.12839897, 0.11902827, 0.10177771]]) 
stats_evening = np.array([[0.19001904, 0.17853512, 0.17442669], [0.13858961, 0.12995696, 0.1299348]])
stats_rainy_noon = np.array([[0.28693173, 0.28569815, 0.29263077], [0.13959084, 0.14426025, 0.15149498]])

Creating the dataframe

# For training set, and for each each reference image (noon), 
# we choose two close cross-domain images (label: 1),
# and two far-away cross-domain images (label: 0)

idx_night_diff_1 = get_diff_image_idx(arr_night_images, len_ds)
idx_night_diff_2 = get_diff_image_idx(arr_night_images, len_ds)
idx_night_same_1 = get_same_image_idx(arr_night_images, len_ds)
idx_night_same_2 = get_same_image_idx(arr_night_images, len_ds)

idx_evening_diff_1 = get_diff_image_idx(arr_evening_images, len_ds)
idx_evening_diff_2 = get_diff_image_idx(arr_evening_images, len_ds)
idx_evening_same_1 = get_same_image_idx(arr_evening_images, len_ds)
idx_evening_same_2 = get_same_image_idx(arr_evening_images, len_ds)

idx_rainy_noon_diff_1 = get_diff_image_idx(arr_rainy_noon_images, len_ds)
idx_rainy_noon_diff_2 = get_diff_image_idx(arr_rainy_noon_images, len_ds)
idx_rainy_noon_same_1 = get_same_image_idx(arr_rainy_noon_images, len_ds)
idx_rainy_noon_same_2 = get_same_image_idx(arr_rainy_noon_images, len_ds)

# For validation set, and for each each reference image (noon),
# we choose one night image of a different run with same idx as label: 1,
# and one night image of different idx as label: 0
idx_night_2_diff = get_diff_image_idx(arr_night_2_images, len_ds)
idx_noon = idx_night_2 = list(range(len_ds))

col_noon = idx_noon + idx_noon + idx_noon + idx_noon + idx_noon + idx_noon
col_night = idx_night_same_1 + idx_night_same_2 + idx_night_diff_1 + idx_night_diff_2 + idx_night_2 + idx_night_2_diff
col_evening = idx_evening_same_1 + idx_evening_same_2 + idx_evening_diff_1 + idx_evening_diff_2 + [None] * 2 * len_ds
col_rainy_noon = idx_rainy_noon_same_1 + idx_rainy_noon_same_2 + idx_rainy_noon_diff_1 + idx_rainy_noon_diff_2 + [None] * 2 * len_ds
col_label = [1] * 2 * len_ds + [0] * 2 * len_ds + [1] *  len_ds + [0] * len_ds

valid_idxs = np.arange(4*len_ds, len(col_night))
col_is_valid = np.array([False] * len_ds * 6)
col_is_valid[valid_idxs] = True

dict_df_night = {'noon'    : col_noon,
                 'night'   : col_night,
                 'label'   : col_label,
                 'is_valid': col_is_valid}

dict_df_evening = {'noon'    : col_noon,
                   'evening'   : col_evening,
                   'label'   : col_label,
                   'is_valid': col_is_valid}

dict_df_rainy_noon = {'noon'    : col_noon,
                      'rainy_noon'   : col_rainy_noon,
                      'label'   : col_label,
                      'is_valid': col_is_valid}

df_night = pd.DataFrame(dict_df_night)
df_evening = pd.DataFrame(dict_df_evening)
df_rainy_noon = pd.DataFrame(dict_df_rainy_noon)

df_night.tail()

	noon	night	label	is_valid
8125	1350	597	0	True
8126	1351	1243	0	True
8127	1352	530	0	True
8128	1353	492	0	True
8129	1354	75	0	True

Creating Dataloaders

Augmentations

# All the transforms for static images:
# It must be as static as possible
transforms_static = albu.Compose([
    albu.Resize(224, 224),
])

# All the transforms for dynamic images:
transforms_dynamic = albu.Compose([
    albu.Resize(224, 224),
    albu.RandomBrightnessContrast(always_apply=True),
    albu.IAAAdditiveGaussianNoise(scale=(0, 0.01*255)),
    albu.ShiftScaleRotate(rotate_limit=10, always_apply=True),
])

transforms_dynamic_xtra = transforms.Compose([
    PerspectiveTransform(pt=0.15)                                    
])

Loading noon feature vectors

Load model

embedding_net = EmbeddingNet()
model = TripletNet(embedding_net)

path_model = os.path.join(path_models, 'static_extractor')
checkpoint = torch.load(path_model, map_location=torch.device('cpu'))
model.load_state_dict(checkpoint['model_state_dict'])

<All keys matched successfully>

Load embeddings (uncomment when embeddings available)

#embeddings = np.load(path_embeddings+'/static_embeddings.npy')

Generate embeddings

embeddings = generate_static_embeddings(model, arr_noon_images, embedding_size=512)

100%|██████████| 1355/1355 [01:11<00:00, 18.86it/s]

Verification via PCA

plot_pca(embeddings)

<Figure size 720x720 with 0 Axes>

If the interactive plot is not viewable please check train_dynamic.html in the html folder.

Save embeddings

path_embeddings_npy = os.path.join(path_embeddings, 'static_embeddings')
np.save(path_embeddings_npy, embeddings)

Defining DataLoader

dict_df_train = {"night": df_night[df_night['is_valid']==False],
                 "evening": df_evening[df_evening['is_valid']==False],
                 "rainy_noon": df_rainy_noon[df_rainy_noon['is_valid']==False]}
dict_df_valid = {"night": df_night[df_night['is_valid']==True]}

dict_df_arr_imgs_train = {"static": arr_noon_images,
                          "night": arr_night_images,
                          "evening": arr_evening_images,
                          "rainy_noon": arr_rainy_noon_images}
dict_df_arr_imgs_valid = {"static": arr_noon_images,
                          "night": arr_night_2_images}

dict_stats_train = {"static": stats_noon,
                    "night": stats_night,
                    "evening": stats_evening,
                    "rainy_noon": stats_rainy_noon}
dict_stats_valid = {"static": stats_noon,
                    "night": stats_night}

train_ds = CrossDomainDataset(dict_df_train,
                              embeddings,
                              dict_df_arr_imgs_train,
                              dict_stats_train,
                              static_tfm=transforms_static,
                              dynamic_tfm=transforms_dynamic,
                              xtra_tfm=transforms_dynamic_xtra)

valid_ds = CrossDomainDataset(dict_df_valid,
                              embeddings,
                              dict_df_arr_imgs_valid,
                              dict_stats_valid,
                              static_tfm=transforms_static,
                              dynamic_tfm=transforms_static)

bs = 128
cuda = torch.cuda.is_available()
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
train_dl = DataLoader(train_ds, batch_size=bs, shuffle=True)
valid_dl = DataLoader(valid_ds, batch_size=bs, shuffle=False)

data = {
    'train': train_dl,
    'validation': valid_dl
}

Plotting examples

domain = "rainy_noon"
stats_static, stats_domain = train_ds.get_domain_stats(domain)
stats_static = torch.Tensor([stats_static[0][..., None, None], stats_static[1][..., None, None]])
stats_domain = torch.Tensor([stats_domain[0][..., None, None], stats_domain[1][..., None, None]])
plot_crossdomain_ex(train_ds, stats_static, stats_domain, rows=2, domain=domain)

domain = "night"
stats_static, stats_domain = train_ds.get_domain_stats(domain)
stats_static = torch.Tensor([stats_static[0][..., None, None], stats_static[1][..., None, None]])
stats_domain = torch.Tensor([stats_domain[0][..., None, None], stats_domain[1][..., None, None]])
plot_crossdomain_ex(train_ds, stats_static, stats_domain, rows=2, domain=domain)

domain = "evening"
stats_static, stats_domain = train_ds.get_domain_stats(domain)
stats_static = torch.Tensor([stats_static[0][..., None, None], stats_static[1][..., None, None]])
stats_domain = torch.Tensor([stats_domain[0][..., None, None], stats_domain[1][..., None, None]])
plot_crossdomain_ex(train_ds, stats_static, stats_domain, rows=2, domain=domain)

Model and training loop

gc.collect()
use_cuda = True
model = DynamicNet(data, ['night', 'evening', 'rainy_noon'])

Keys of dynamic extractors: dict_keys(['night', 'evening', 'rainy_noon'])

Load model (uncomment when a model is available)

path_folder_model = os.path.join(path_models, 'multi-domain-test')
model.load(path_folder_model, flag_inference=True)

Loaded from data/saved/models/multi-domain-test sucessfully!

Training and Saving

model.fit(max_lr=1e-3, epochs=5, continue_train=False)

Saving

path_folder_model = os.path.join(path_models, 'multi-domain-test')

model.save(path_folder_model)

data/saved/models/multi-domain-test already exists. Delete it?(y/n)
y
Saved at data/saved/models/multi-domain-test successfully!

Proof of Concept (for rotating training procedure)

# import torch
# import torch.optim

# day = torch.tensor([1.0, 2.0], requires_grad=True)
# night = torch.tensor([1.0, 2.0], requires_grad=True)
# evening = torch.tensor([1.0, 2.0], requires_grad=True)
# classifier = torch.tensor([5.0, 10.0], requires_grad=True)

# opt_night = optim.SGD([night], lr=1)
# opt_evening = optim.SGD([evening], lr=1)
# opt_static = optim.SGD([day, classifier], lr=2)

# opt_static.param_groups

[{'dampening': 0,
  'lr': 2,
  'momentum': 0,
  'nesterov': False,
  'params': [tensor([1., 2.], requires_grad=True),
   tensor([ 5., 10.], requires_grad=True)],
  'weight_decay': 0}]

# day, night, evening, classifier

(tensor([1., 2.], requires_grad=True),
 tensor([1., 2.], requires_grad=True),
 tensor([1., 2.], requires_grad=True),
 tensor([ 5., 10.], requires_grad=True))

# day.grad, night.grad, evening.grad, classifier.grad

(None, None, None, None)

# Day and night pass 
# mul = day * night
# loss = (mul * classifier).sum()

# loss.backward()

# day.grad, night.grad, evening.grad, classifier.grad

(tensor([ 5., 20.]), tensor([ 5., 20.]), None, tensor([1., 4.]))

# opt_night.step()

# day, night, evening, classifier

(tensor([1., 2.], requires_grad=True),
 tensor([ -4., -18.], requires_grad=True),
 tensor([1., 2.], requires_grad=True),
 tensor([ 5., 10.], requires_grad=True))

# day.grad, night.grad, evening.grad, classifier.grad

(tensor([ 5., 20.]), tensor([ 5., 20.]), None, tensor([1., 4.]))

# Day and evening pass 
# mul = day * evening
# loss = (mul * classifier).sum()

# loss.backward()

# day.grad, night.grad, evening.grad, classifier.grad

(tensor([10., 40.]), tensor([ 5., 20.]), tensor([ 5., 20.]), tensor([2., 8.]))

# opt_evening.step()

# day, night, evening, classifier

(tensor([1., 2.], requires_grad=True),
 tensor([ -4., -18.], requires_grad=True),
 tensor([ -4., -18.], requires_grad=True),
 tensor([ 5., 10.], requires_grad=True))

# day.grad.div_(2)
# classifier.grad.div_(2)

tensor([1., 4.])

# opt_static.step()

# day, night, evening, classifier

(tensor([ -9., -38.], requires_grad=True),
 tensor([ -4., -18.], requires_grad=True),
 tensor([ -4., -18.], requires_grad=True),
 tensor([3., 2.], requires_grad=True))

# day.grad, night.grad, evening.grad, classifier.grad

(tensor([ 5., 20.]), tensor([ 5., 20.]), tensor([ 5., 20.]), tensor([1., 4.]))

# opt_evening.zero_grad()
# opt_night.zero_grad()
# opt_static.zero_grad()

# day.grad, night.grad, evening.grad, classifier.grad

(tensor([0., 0.]), tensor([0., 0.]), tensor([0., 0.]), tensor([0., 0.]))